import requests, re, pymysql
import concurrent.futures
from dbutils.pooled_db import PooledDB
import sys

token = "20241113LNRuLTdDpIVgBuf9sHX"
cookie = "acw_tc=0b63bb3617315433846626650e5b272118e0992e6a9eedb6cc0e747b0fbee2; SERVERID=bb67fa45d81410ac7830efa9e139446e|1731543384|1731543001"
dev_token = "AiM1dyZ-ZP_Fh0AFpaIZdDFu7J0zcYcydbR-MXRYqVkE"
businessType = 3 #1樊登读书 2非凡精读 3李蕾讲经典
headers = {
    "User-Agent": "KKWeight_Android",
    "X-DUSHU-APP-PLT": "2",
    "X-DUSHU-BUILD-IDENTIFIER": "com.kebida.dushu",
    "X-DUSHU-APP-VER": "6.7.6",
    "X-DUSHU-APP-SYSVER": "13",
    "X-DUSHU-APP-DEVTOKEN": dev_token,
    "X-DUSHU-APP-CHN": "yingyongbao",
    "Content-Type": "application/json;charset=UTF-8",
    "Cookie": cookie
}


def remove_html_tags(text):
    # 定义正则表达式模式，去除HTML标签、JavaScript代码块和CSS样式等
    pattern = r'<!--.*?-->|<script.*?>.*?</script>|<style.*?>.*?</style>|<[^>]+>'
    # 使用sub方法替换所有匹配的模式为空字符串，去除HTML标签、CSS和JavaScript代码
    cleaned_text = re.sub(pattern, '', text, flags=re.DOTALL)
    # 查找并删除"读完本文约需35分钟"之前的所有内容
    match = re.search(r'读完本文约需.*', cleaned_text)
    if match:
        cleaned_text = match.group(0)
    return cleaned_text

# 获取图片链接
def get_image_link(content):
    pattern = r'<img src="([^"]*)">'
    match = re.search(pattern, content)
    if match:
        return match.group(1)
    else:
        return None


def get_book_info(bookId):
    url = "http://gw1-ipv6.dushu365.com/resource-orchestration-system/book/v101/content"
    data = {
        "appId": "2001",
        "token": token,
        "bookId": bookId
    }
    response = requests.post(url, headers=headers, json=data)
    # 检查请求是否成功
    if response.status_code == 200:
        data = response.json()
        return data['data']  # 返回JSON格式的响应内容
    else:
        return f"Error: {response.status_code}"


def get_lists():
    url = "https://gw1-ipv6.dushu365.com/resource-orchestration-system/book/classify/v100/listClassifyBook"
    data = {
        "publishYear": 0,
        "sortType": 1,
        "classifyIds": [],  # [10054]亲子关系
        "pageNo": 1,
        "appId": "2001",
        "pageSize": 600,
        "businessType": businessType, 
        "token": token,
        "bookReadStatus": -1
    }
    response = requests.post(url, headers=headers, json=data)
    # 检查请求是否成功
    if response.status_code == 200:
        return response.json()  # 返回JSON格式的响应内容
    else:
        return f"Error: {response.status_code}"


def get_book_text(fragmentId):
    url = "https://gw1-ipv6.dushu365.com/resource-orchestration-system/book/v101/content"
    data = {
        "appId": "2001",
        "fragmentId": fragmentId,
        "prekey": "",
        "token": token
    }
    response = requests.post(url, headers=headers, json=data)
    # 检查请求是否成功
    if response.status_code == 200:
        data = response.json()
        # data = remove_html_tags(data['data']['content'])
        return data['data']['content']  # 返回JSON格式的响应内容
    else:
        return f"Error: {response.status_code}"

def get_book_mind(fragmentId):
    url = "https://gw1-ipv6.dushu365.com/resource-orchestration-system/book/v101/content"
    data = {
        "appId": "2001",
        "fragmentId": fragmentId,
        "prekey": "",
        "token": token
    }
    response = requests.post(url, headers=headers, json=data)
    # 检查请求是否成功
    if response.status_code == 200:
        data = response.json()
        image_link = get_image_link(data['data']['content'])
        print(image_link if image_link else data)
        return image_link  # 返回图片链接或None
    else:
        return f"Error: {response.status_code}"

# 插入数据函数
# 插入或更新书籍信息到数据库
def save_book_to_db(book):
    try:
        connection = pool.connection()
        with connection.cursor() as cursor:
            # 先检查bookid是否存在
            check_sql = "SELECT COUNT(*) FROM books WHERE bookid = %s"
            cursor.execute(check_sql, (book['bookid'],))
            exists = cursor.fetchone()['COUNT(*)'] > 0

            if exists:
                # 如果存在，更新数据
                sql = """
                UPDATE books SET 
                    title=%s, 
                    author=%s, 
                    bookCoverUrl=%s, 
                    speaker=%s, 
                    bookCategoryName=%s, 
                    recommendInfo=%s, 
                    audioUrl=%s, 
                    videoUrl=%s, 
                    fragmentId=%s, 
                    bookText=%s,
                    publishTime=%s,
                    score=%s,
                    videoCoverUrl=%s,
                    mindMapUrl=%s,
                    businessType=%s,
                    readCount=%s,
                    playNum=%s
                WHERE bookid=%s
                """
                cursor.execute(sql, (
                    book['title'],
                    book['author'],
                    book['bookCoverUrl'],
                    book['speaker'],
                    book['bookCategoryName'],
                    book['recommendInfo'],
                    book['audioUrl'],
                    book['videoUrl'],
                    book.get('fragmentId', None),
                    book['bookText'],
                    int(book.get('publishTime', 0)) if book.get('publishTime') else None,
                    int(book.get('score', 0)) if book.get('score') else None,
                    book.get('videoCoverUrl', None),
                    book.get('mindMapUrl', None),
                    book.get('businessType', None),
                    book.get('readCount', None),
                    book.get('playNum', None),
                    book['bookid']
                ))
            else:
                # 如果不存在，插入数据
                sql = """
                INSERT INTO books (bookid, title, author, bookCoverUrl, speaker, bookCategoryName, recommendInfo, audioUrl, videoUrl, fragmentId, bookText, publishTime, score, videoCoverUrl, mindMapUrl, businessType, readCount, playNum)
                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                """
                cursor.execute(sql, (
                    book['bookid'],
                    book['title'],
                    book['author'],
                    book['bookCoverUrl'],
                    book['speaker'],
                    book['bookCategoryName'],
                    book['recommendInfo'],
                    book['audioUrl'],
                    book['videoUrl'],
                    book.get('fragmentId', None),
                    book['bookText'],
                    int(book.get('publishTime', 0)) if book.get('publishTime') else None,
                    int(book.get('score', 0)) if book.get('score') else None,
                    book.get('videoCoverUrl', None),
                    book.get('mindMapUrl', None),
                    book.get('businessType', None),
                    book.get('readCount', None),
                    book.get('playNum', None)
                ))
        connection.commit()
        print(f"Book {book['title']} (ID: {book['bookid']}) 保存成功")
    except Exception as e:
        print(f"Book {book['title']} (ID: {book['bookid']}) 保存失败: {e}")
        connection.rollback()
    finally:
        connection.close()

# 连接到 MySQL 数据库
# 创建数据库连接池
pool = PooledDB(
    creator=pymysql,  # 使用 pymysql 作为连接库
    maxconnections=10,  # 连接池最大连接数
    mincached=2,  # 初始化时的连接数
    blocking=True,  # 当连接池没有可用连接时等待
    host='localhost',
    user='root',
    password='wxmp2014',
    database='fdds',
    charset='utf8mb4',
    cursorclass=pymysql.cursors.DictCursor
)

# 调用函数
response_data = get_lists()
print("一共获取：", len(response_data['data']), "本书")
books_data = []
# import json
# print(json.dumps(response_data, indent=4, ensure_ascii=False))
# sys.exit()
# book_details = get_book_info("400119271")
# print(book_details)


# 处理书籍的函数
def process_book(item):
    try:
        bookinfo = get_book_info(item["id"])
        book = {}
        book['bookid'] = item["id"]
        book['title'] = item["title"]
        book['author'] = bookinfo['bookInfo']['author']
        book['bookCoverUrl'] = bookinfo['bookInfo']['bookCoverUrl']
        book['speaker'] = bookinfo['bookInfo']['speaker']
        book['publishTime'] = int(bookinfo['bookInfo']['publishTime'])/1000
        book['score'] = bookinfo['bookInfo']['score']
        book['bookCategoryName'] = bookinfo['bookInfo']['bookCategoryName']
        book['recommendInfo'] = bookinfo['recommendVO'].get('recommendInfo', '')
        book['audioUrl'] = bookinfo['audioInfo']['audioUrl']
        book['businessType'] = businessType
        book['readCount'] = item['readCount']
        book['playNum'] = bookinfo['bookInfo']['playNum']
        fragmentId = None
        mindId = None

        for i in bookinfo.get('articles', []):
            if i.get('moduleName') == "文字稿":
                fragmentId = i.get('fragmentId')
            elif i.get('moduleName') == "思维导图":
                mindId = i.get('fragmentId')
        if fragmentId is None:
            print(f"Book {book['title']} has no fragmentId for 文字稿")
            return

        book['fragmentId'] = fragmentId

        if 'content' in bookinfo and businessType == 1:
            if mindId is not None:
                book['mindMapUrl'] = get_book_mind(mindId)
            else:
                print(f"Book {book['title']} has no mindId for 思维导图")
                return
        elif businessType == 3:
            book['mindMapUrl'] = ""
        

        if not book.get('mindMapUrl') and businessType == 1:
            print(f"Retrying to get mindMapUrl for {book['title']}")
            if mindId is not None:
                book['mindMapUrl'] = get_book_mind(mindId)

        if 'videoInfo' in bookinfo:
            book['videoUrl'] = bookinfo['videoInfo'].get('videoUrl', None)
            book['videoCoverUrl'] = bookinfo['videoInfo'].get('cover', None)
        else:
            book['videoUrl'] = None

        if fragmentId is None:
            book['bookText'] = ""
        else:
            book['bookText'] = get_book_text(book['fragmentId'])
        
        
        # 保存当前书籍信息到数据库
        save_book_to_db(book)
    except Exception as e:
        print(f"Error processing book {item['id']}{book['title']}: {e}")

# 使用 ThreadPoolExecutor 进行多线程处理
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
    executor.map(process_book, response_data['data'])
